package com.eliansun.crawler.pageparse.jiangxi;

import com.eliansun.crawler.pageparse.AbstractPageProcess;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.selector.Selectable;

import java.io.IOException;
import java.net.URL;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

/**
 * user: lxc
 * data:2019- 07-03-14
 * 九真健康网 www.jiuzhenjk.com
 */
public class Jiuzhen extends AbstractPageProcess {

    int count = 0;

    @Override
    public void process(Page page) {

        try {
            //列表界面
            if (page.getUrl().toString().indexOf("search") != -1) {
                List<Selectable> nodes = page.getHtml().xpath("//div[@class=goods-list]//div[@class=img]/a").nodes();
                if (nodes.size() == 0) {
                    return;
                }

                //列表界面解析出对应的详情url，放到队列中
                for (Selectable node : nodes) {
                    String s = node.links().toString();
                    page.addTargetRequest(s);
                }

                Document document = Jsoup.parse(new URL(page.getUrl().toString()), 5000);
                Elements scripts = document.select("script");
                String a = "";
                for (Element script : scripts) {
                    if (script.html().contains("pages")) {
                        a = script.
                                toString().
                                replace("\r\n", "").
                                trim().
                                split(",")[3].
                                trim().
                                split(":")[1].
                                trim();
                    }
                }

                //获取下一页放到队列中
                if (!a.equals("")) {
                    //总页数
                    Integer totalPage = (Integer.parseInt(a));
                    for (int i = 1; i < totalPage; i++) {
                        String url = page.getUrl().toString();
                        page.addTargetRequest(url.replace("page=1", "page=" + (i + 1)));
                    }
                }
            } else {
                //详情界面

                Map<String, Object> resMap = new HashMap<>();

                // 商品名称
                resMap.put("goodsName", page.getHtml().xpath("//div[@class=intro-name]/h2/text()").get());

                //价格
                String price = page.getHtml().xpath("//div[@id='j-shop-price']/text()").get();
                resMap.put("price", price.replace("￥", ""));

                Map<String, Object> resultMap = new HashMap<>();
                resultMap.put("spec", "");
                resultMap.put("medicalDeviceNo", "");
                resultMap.put("productCompany", "");
                resultMap.put("price", resMap.get("price"));
                resultMap.put("brand", "");
                resultMap.put("productName", resMap.get("goodsName"));
                resultMap.put("executiveStandard", "");
                resultMap.put("nationality", "");
                resultMap.put("store", super.cTerrace);
                resultMap.put("url", page.getUrl());
                resultMap.put("version", "");
                page.putField("result", resultMap);

                count++;
                System.out.println(count);
            }
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    Site site = Site.me()
            //设置编码
            .setCharset("utf-8")
            //设置抓取间隔时间
            .setSleepTime(1000 * 3)
            //设置超时时间，单位毫秒
            .setTimeOut(1000 * 3)
            //设置重试间隔时间
            .setRetrySleepTime(1000 * 3)
            //设置重试次数
            .setRetryTimes(3);

    @Override
    public Site getSite() {
        return site;
    }


    public static void main(String[] args) {
        Spider.create(new Jiuzhen())
                .addUrl("http://www.jiuzhenjk.com/index.php/goods-3054.html")
                //设置多个线程同时抓取数据
                .thread(1)
                .run();

    }

}
